# -*- coding: utf-8 -*-
# Program function：
import pandas as pd
import matplotlib
from sklearn.model_selection import train_test_split
import re
# 设置后端
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
from imblearn.over_sampling import RandomOverSampler
# 导入label_Encoder
from sklearn.preprocessing import LabelEncoder
import torch
ros = RandomOverSampler(random_state=1)


def guolv(path):
    df = pd.read_csv('../dm00_data/train.txt', sep='\t' ,encoding='utf-8',names=['department','ask'])
    # print(len(df['department'].unique()))
    # print(len(df))
    #把department 热编码
    # le = LabelEncoder()
    # df['department'] = le.fit_transform(df['department'])
    # print(df['department'])
    # class2num = {le.classes_[i]:i for i in range(len(le.classes_))}
    # num2class = {i:le.classes_[i] for i in range(len(le.classes_))}
    # def remove_all_nontext(text):
    #     # 移除Emoji+颜文字(如^^, ^_^)
    #     emoji_pattern = re.compile("["
    #         u"\U0001F600-\U0001F64F"
    #         u"\U0001F300-\U0001F5FF"
    #         "]+", flags=re.UNICODE)
    #     kaomoji_pattern = re.compile(r'\^[\^_]+?\^')  # 匹配^_^
    #     text = emoji_pattern.sub('', text)
    #     return kaomoji_pattern.sub('', text)
    #
    # df['ask'] = df['ask'].apply(lambda x:remove_all_nontext(x))
    # df['ask'] = df['ask'].apply(lambda x:re.sub(r'\s+', ' ', x))
    # 过滤掉ask重复的行
    df.drop_duplicates(subset=['ask'], inplace=True)
    # x = df.groupby('department').count().sort_values('ask', ascending=False)
    # print(x)
    # 打印类别
    print(len(df['department'].unique()))
    print(len(df))
    # 打印ask的内容长度分布
    # ask_len = df['ask'].str.len().value_counts()
    # print(ask_len)
    # 打印类别数量
    ask_len = df['department'].value_counts()
    print(ask_len)
    # 过滤类别数量小于100
    df = df[df['department'].map(df['department'].value_counts()) > 100]
    print(len(df['department'].unique()))



# 过采样
# x_resampled, y_resampled = ros.fit_resample(df[['ask']], df['department'])
#
# X_train, X_temp, y_train, y_temp = train_test_split(
#     x_resampled, y_resampled,
#     test_size=0.2,           # 临时分出20%
#     stratify=y_resampled, # 按y分层抽样
#     random_state=1       # 随机种子
# )
#
# # 第二次拆分：将20%的数据再均分为验证集和测试集（各占10%）
# X_val, X_test, y_val, y_test = train_test_split(
#     X_temp, y_temp,
#     test_size=0.5,           # 对20%的数据再对半分
#     stratify=y_temp,        # 继续分层抽样
#     random_state=1
# )
# #
# train_df = pd.concat([X_train, y_train], axis=1)
# dev_df = pd.concat([X_val, y_val], axis=1)
# test_df = pd.concat([X_test, y_test], axis=1)
# #
# train_df.to_csv('../dm00_data/train.csv', index=False, header=False, sep='\t')
# dev_df.to_csv('../dm00_data/dev.csv', index=False, header=False, sep='\t')
# test_df.to_csv('../dm00_data/test.csv', index=False, header=False, sep='\t')
# # #
# # # 保存class2num
# with open('../dm00_data/class2num.txt', 'w') as f:
#     for k, v in class2num.items():
#         f.write(f'{k}:{v}\n')
#
#
# def data_load(path,name):
#     df = pd.read_csv(path, sep='\t', encoding='utf-8',names=['text','lable'])
#     df.drop_duplicates(subset=['text'], inplace=True)
#     df.to_csv(f'../dm00_data/{name}.txt', index=False, header=False, sep='\t')
#     return df
#
# data_load('../dm00_data/train.csv','train2')
# data_load('../dm00_data/test.csv','test2')
# data_load('../dm00_data/dev.csv','dev2')


# for i in range(len(x_resampled)):
#     print(num2class[x_resampled.iloc[i][0]], y_resampled.iloc[i])
#     if i == 10:
#         break


# 画箱型图 标注中位线等
# plt.figure(figsize=(10, 5))
# plt.boxplot(ask_len.values,showcaps = True)
# plt.show()
